package com.coderside.fragment.netWork;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.httpclient.HttpClient;

/**
 * 爬取并分析ITEYE论坛的文章
 * 并以html的格式保存到本地文件夹中
 * html页面分析：iteye论坛的页面
 * 
 * @author admin
 *
 */
public class ParseIteye {
	private static String defaultEncode = "UTF-8";

	private static String dirPath = "D:\\htmlParse";
	private static String baseUrl = "http://www.iteye.com";
	private static Map<String, Pattern> patternMap = new HashMap<String, Pattern>();

	/**
	* 测试
	*
	* @param args
	*/
	public static void main(String[] args) {
		//System.setProperty("http.proxyHost", "165.228.128.10");
		//System.setProperty("http.proxyPort", "3128");
		//System.setProperty("http.proxySet", "true");

		/***当你爬取的内容过于频繁的时候、Iteye会要求输入一个验证码，导致运行异常**/
		String reqUrl = null;
		long a = System.currentTimeMillis();
		for (int i = 1; i <= 103; i++) {
			reqUrl = "http://www.iteye.com/forums/tag/Java?page=" + i;
			parseforums(reqUrl);
		}
		long b = System.currentTimeMillis();
		System.out.println(b - a);
	}

	/**
	 * 保存正则表达式到缓存中并返回
	 * 下次可以直接返回 不用重复编译
	 * PATTERNMODE 为正则表达式的模式参数 e.g:Pattern.CASE_INSENSITIVE
	 * @param regex
	 * @param PATTERNMODE 为-1的时候说明不需要该参数
	 * @return
	 */
	public static Pattern cachePattern(String regex, int PATTERNMODE) {
		Pattern pattern = patternMap.get(regex);
		if (null != pattern) {
			return pattern;
		}
		Pattern p = null;
		if (PATTERNMODE == -1) {
			p = Pattern.compile(regex);
			patternMap.put(regex, p);
		} else {
			p = Pattern.compile(regex, PATTERNMODE);
			patternMap.put(regex, p);
		}
		return p;
	}

	/**
	 *提取有用的A标签 可以再递归捉取
	 * @param tr
	 * @return
	 */
	public static List<Link> pickupLink(String td) {

		List<Link> resultList = new ArrayList<Link>();
		String regex = "(?i)<[a]\\s*(href=[^>]+)>(.*?)</[a]>";
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<table .+?>开头，</table>结尾的文档
		Matcher m = p.matcher(td);//开始编译
		while (m.find()) {
			Link l = new Link();
			l.setHost(baseUrl);
			String temp = m.group(1);
			temp = temp.replace("href=", "");
			temp = temp.replace("\"", "");
			temp = temp.replace("\'", "");
			if (temp.indexOf("page=1") != -1) {
				continue;
			}
			l.setAction(temp);
			l.setText(m.group(2));
			resultList.add(l);
			parseTopic(l.getHost() + l.getAction());
		}
		System.out.println(resultList);
		return resultList;

	}

	/**
	 *每行有多列 把我们需要的列提取出来
	 * @param tr
	 * @return
	 */
	public static List<String> pickupTd(String tr) {

		List<String> resultList = new ArrayList<String>();
		String regex = "(?i)<td\\s*?class=\"topic_title\">([.*?\\s\\S*?]+?)</td>";
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<table .+?>开头，</table>结尾的文档
		Matcher m = p.matcher(tr);//开始编译
		while (m.find()) {
			resultList.add(m.group(1));//获取被匹配的部分
			pickupLink(m.group(1));
		}
		//		System.out.println(resultList);
		return resultList;
	}

	/**
	 * 每个table的内容有多行(tr)
	 * 使用正则多行匹配table标签内的内容
	 * @param body
	 */
	public static List<String> pickupTr(String table) {
		List<String> resultList = new ArrayList<String>();
		String regex = "(?i)<tr>([.+\\s\\S*]+?)</tr>";
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<table .+?>开头，</table>结尾的文档
		Matcher m = p.matcher(table);//开始编译
		while (m.find()) {
			resultList.add(m.group(1));
			pickupTd(m.group(1));
		}
		return resultList;
	}

	/**
	 * 每页内容有一个table
	 * 使用正则多行匹配table标签内的内容
	 * @param body
	 */
	public static String pickupTable(String body) {
		String regex = "(?i)<table\\s.+>([.+\\s\\S*]+?)</table>";
		Pattern p = cachePattern(regex, Pattern.MULTILINE);//匹配<table .+?>开头，</table>结尾的文档
		Matcher m = p.matcher(body);//开始编译
		if (m.find()) {
			pickupTr(m.group(1));
			return m.group(1);
		}
		return "";
	}

	/**
	 * 解析论坛主页 获取主页上的链接
	 * @param reqUrl
	 */
	public static void parseforums(String reqUrl) {

		//System.setProperty("http.proxyHost", "165.228.128.10");
		//System.setProperty("http.proxyPort", "3128");
		//System.setProperty("http.proxySet", "true");

		//		reqUrl = "http://www.iteye.com/forums/tag/Java";
		Map<String, String> headers = new HashMap<String, String>();
		headers.put("Accept-Encoding", "gzip,deflate");

		HttpClient httpClient = HttpClientHelper.getHttpClient();
		String htmlContent = HttpClientHelper.doRequest(httpClient, reqUrl, headers, null, defaultEncode);
		pickupTable(htmlContent);
	}

	public static void parseTopic(String reqUrl) {
		Map<String, String> headers = new HashMap<String, String>();
		headers.put("Accept-Encoding", "gzip,deflate");
		HttpClient httpClient = HttpClientHelper.getHttpClient();
		String htmlContent = HttpClientHelper.doRequest(httpClient, reqUrl, headers, null, defaultEncode);
		File file = new File(dirPath);
		if (!file.exists()) {
			file.mkdir();
		}
		String f = getSuffixName(reqUrl, baseUrl);
		f = f.replaceAll("\\?", "_");
		file = new File(dirPath + "\\" + f + ".html");
		try {
			FileOutputStream fos = new FileOutputStream(file);
			fos.write(htmlContent.getBytes(defaultEncode));
			fos.flush();
			fos.close();
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	private static String getSuffixName(String reqUrl, String baseUrl) {
		int temp = reqUrl.indexOf(baseUrl);
		String[] a = reqUrl.substring(temp).split("/");
		return a[a.length - 1];
	}

	static class Link {

		String host;
		String action;
		String text;

		@Override
		public String toString() {
			return host + action + ":" + text;
		}

		public String getHost() {
			return host;
		}

		public void setHost(String host) {
			this.host = host;
		}

		public String getAction() {
			return action;
		}

		public void setAction(String action) {
			this.action = action;
		}

		public String getText() {
			return text;
		}

		public void setText(String text) {
			this.text = text;
		}

	}
}
